#! python3
# 下载阳光电影的高分经典电影

import requests
import bs4
import os
import logging
import mysqlConnHelper

# 根据视频地址，使用urllib.request.urlretrieve()即可将视频下载下来
from urllib import request

logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s - %(message)s')
logging.disable()

mysqlHelper = mysqlConnHelper.mysqlHelper('localhost', '3306', 'root', 'root', 'runoob_db')
"""
CREATE TABLE `movies` (
   `id`     int(11) NOT NULL AUTO_INCREMENT,
   `url`    varchar(255) DEFAULT NULL,
   `status` int(11) DEFAULT '0' COMMENT '0-初始，1-已下载，2-下载失败，3-下载中',
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8
"""


def getMovieLinks():
    """
    获取所有页面的电影链接，并保存在movies表中
    :return:
    """
    try:
        BASE_URL = 'https://www.ygdy8.com/html/gndy/jddy/20160320/'
        page_url = '50541.html'  # 第一页

        previous_page = ''  # 上一页

        # 获取所有页面内的电影下载地址
        moviesLinks = list()
        i = 1
        while True:
            url = BASE_URL + page_url
            if url == previous_page:
                # print("第%d页为最后一页" % (i - 1))
                logging.debug("第%d页为最后一页" % (i - 1))
                break
            previous_page = url

            logging.debug("开始获取第" + str(i) + "页的电影列表，地址为：" + url)
            # print('第%d页' % i, url)
            res = requests.get(url)
            res.raise_for_status()
            res.encoding = 'gbk'

            soup = bs4.BeautifulSoup(res.text, features="html.parser")

            elementsMovs = soup.select('p > a')  # <p>下的<a>元素,中间没有其他元素
            for elementsMov in elementsMovs:
                # print(elementsMov.get('href'))
                href = elementsMov.get('href')
                res = mysqlHelper.queryData('select * from movies where url=%s', (href,))
                if len(res) == 0:
                    moviesLink = (elementsMov.get('href'),)
                    moviesLinks.append(moviesLink)
                else:
                    continue

            nextPageLink = soup.select('center > a')  # <center>下的<a>元素,中间没有其他元素
            for page in nextPageLink:
                if page.getText() == '下一页':
                    page_url = page.get('href')
                    break
                else:
                    continue
            i += 1

        if len(moviesLinks) > 0:
            mysqlHelper.insertDatas("INSERT INTO movies (url) VALUES (%s)", moviesLinks)

        return len(moviesLinks)

    except Exception as e:
        # print('there have a error : ' + e)
        logging.debug('there have a error : ' + e)


def mkdir(path_):
    if not os.path.exists(path_):
        os.mkdir(path_)


def downLoadMovies(path_):
    movies = mysqlHelper.queryData("select * from movies where status=%s and id=2", (0,))
    for i in range(len(movies)):
        movie = movies[i]

        id = movie[0]
        url = movie[1]
        # status: 0 - 初始，1 - 已下载，2 - 下载失败，3 - 下载中
        mysqlHelper.execute("update movies set status=%s where id=%s", ('3', id))

        res = requests.get(url)
        res.raise_for_status()
        res.encoding = 'gbk'
        soup = bs4.BeautifulSoup(res.text, features="html.parser")
        elements = soup.select('td > a')  # <p>下的<a>元素,中间没有其他元素

        movieUrl = elements[0].get('href')
        fileName = os.path.basename(movieUrl)
        print(movieUrl)

        #  下载方法一
        # 此处观察movieUrl发现为FTP协议，以下两种下载方法不使用，需要借助下载软件才能继续下载
        # resMovie = requests.get(movieUrl)
        # resMovie.raise_for_status()
        # file = open(os.path.join(path_, fileName), 'wb')
        # for text in resMovie.iter_content(100000):
        #     file.write(text)
        # file.close()

        #  下载方法二
        """
        函数：urllib.urlretrieve(url[, filename[, reporthook[, data]]]) 
        参数说明： 
        url：外部或者本地url ,url中不要含有中文，好像会出错。
        filename：指定了保存到本地的路径； 
        reporthook：是一个回调函数，当连接上服务器、以及相应的数据块传输完毕的时候会触发该回调。我们可以利用这个回调函数来显示当前的下载进度。 
        data：指post到服务器的数据。该方法返回一个包含两个元素的元组(filename, headers)，filename表示保存到本地的路径，header表示服务器的响应头。 
        """
        # request.urlretrieve(movieUrl, fileName, fun)

        mysqlHelper.execute("update movies set status=%s where id=%s", ('1', id))


def fun(blockNum, blockSize, totalSize):
    """
    blocknum:当前的块编号
    blocksize:每次传输的块大小
    totalsize:网页文件总大小
    """
    downSIZE = blockNum * blockSize / 1024 / 1024
    totalSIZE = totalSize / 1024 / 1024
    percent = downSIZE / totalSIZE
    if percent > 1.0:
        percent = 1.0
    percent = percent * 100
    print("[%.2fM/%.2fM]\t%.2f%%" % (downSIZE, totalSIZE, percent))


if __name__ == '__main__':
    # number = getMovieLinks()
    # print(number)

    path = 'F:/download/高分经典'
    mkdir(path)

    downLoadMovies(path)
